{
 "cells": [
  {
   "cell_type": "code",
   "id": "initial_id",
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2025-05-09T05:59:18.664529Z",
     "start_time": "2025-05-09T05:59:18.661528Z"
    }
   },
   "source": "from modelscope import AutoTokenizer",
   "outputs": [],
   "execution_count": 126
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-09T05:59:18.673886Z",
     "start_time": "2025-05-09T05:59:18.670835Z"
    }
   },
   "cell_type": "code",
   "source": [
    "dialog = [{\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n",
    "          {\"role\": \"user\", \"content\": \"天空为什么是蓝色的？\"},\n",
    "          {\"role\": \"assistant\", \"content\": \"这是由于光的散射引起的。\"}]"
   ],
   "id": "16c3fedfca5895b1",
   "outputs": [],
   "execution_count": 127
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-09T05:59:19.761549Z",
     "start_time": "2025-05-09T05:59:18.706595Z"
    }
   },
   "cell_type": "code",
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(\"Qwen/Qwen3-8B\")\n",
    "# print(tokenizer.chat_template)"
   ],
   "id": "4af5728536b824ff",
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-05-09 13:59:19,142 - modelscope - WARNING - Using branch: master as version is unstable, use with caution\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading Model from https://www.modelscope.cn to directory: C:\\Users\\51165\\.cache\\modelscope\\hub\\models\\Qwen\\Qwen3-8B\n"
     ]
    }
   ],
   "execution_count": 128
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-09T05:59:19.798741Z",
     "start_time": "2025-05-09T05:59:19.795907Z"
    }
   },
   "cell_type": "code",
   "source": [
    "input = tokenizer.apply_chat_template(dialog)\n",
    "print(tokenizer.decode(input, skip_special_tokens=False))\n"
   ],
   "id": "7d20ca4cd6af7866",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<|im_start|>system\n",
      "You are a helpful assistant.<|im_end|>\n",
      "<|im_start|>user\n",
      "天空为什么是蓝色的？<|im_end|>\n",
      "<|im_start|>assistant\n",
      "<think>\n",
      "\n",
      "</think>\n",
      "\n",
      "这是由于光的散射引起的。<|im_end|>\n",
      "\n"
     ]
    }
   ],
   "execution_count": 129
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-09T05:59:19.834912Z",
     "start_time": "2025-05-09T05:59:19.831409Z"
    }
   },
   "cell_type": "code",
   "source": [
    "input = tokenizer.apply_chat_template(dialog, return_tensors=\"pt\")\n",
    "print(input)\n",
    "print(tokenizer.batch_decode(input, skip_special_tokens=False))"
   ],
   "id": "7ce02fd7c8670ca3",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[151644,   8948,    198,   2610,    525,    264,  10950,  17847,     13,\n",
      "         151645,    198, 151644,    872,    198, 101916, 100678,  20412, 105681,\n",
      "           9370,  11319, 151645,    198, 151644,  77091,    198, 151667,    271,\n",
      "         151668,    271, 100346, 101887,  99225,   9370,  99632,  99759, 107503,\n",
      "           1773, 151645,    198]])\n",
      "['<|im_start|>system\\nYou are a helpful assistant.<|im_end|>\\n<|im_start|>user\\n天空为什么是蓝色的？<|im_end|>\\n<|im_start|>assistant\\n<think>\\n\\n</think>\\n\\n这是由于光的散射引起的。<|im_end|>\\n']\n"
     ]
    }
   ],
   "execution_count": 130
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-09T05:59:19.874252Z",
     "start_time": "2025-05-09T05:59:19.870743Z"
    }
   },
   "cell_type": "code",
   "source": [
    "print(tokenizer.bos_token)\n",
    "print(tokenizer.eos_token)\n",
    "print(tokenizer.special_tokens_map)"
   ],
   "id": "6d9e55ee89d45eaa",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "None\n",
      "<|im_end|>\n",
      "{'eos_token': '<|im_end|>', 'pad_token': '<|endoftext|>', 'additional_special_tokens': ['<|im_start|>', '<|im_end|>', '<|object_ref_start|>', '<|object_ref_end|>', '<|box_start|>', '<|box_end|>', '<|quad_start|>', '<|quad_end|>', '<|vision_start|>', '<|vision_end|>', '<|vision_pad|>', '<|image_pad|>', '<|video_pad|>']}\n"
     ]
    }
   ],
   "execution_count": 131
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-09T05:59:21.419855Z",
     "start_time": "2025-05-09T05:59:19.908366Z"
    }
   },
   "cell_type": "code",
   "source": "tokenizer2 = AutoTokenizer.from_pretrained(\"LLM-Research/Meta-Llama-3.1-8B-Instruct\")",
   "id": "3533de718ba171cd",
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-05-09 13:59:20,833 - modelscope - WARNING - Using branch: master as version is unstable, use with caution\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading Model from https://www.modelscope.cn to directory: C:\\Users\\51165\\.cache\\modelscope\\hub\\models\\LLM-Research\\Meta-Llama-3.1-8B-Instruct\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-05-09 13:59:21,128 - modelscope - INFO - Target directory already exists, skipping creation.\n"
     ]
    }
   ],
   "execution_count": 132
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-09T05:59:21.490200Z",
     "start_time": "2025-05-09T05:59:21.486197Z"
    }
   },
   "cell_type": "code",
   "source": [
    "input2 = tokenizer2.apply_chat_template(dialog, return_tensors=\"pt\", return_dict=True,)\n",
    "print(input2)\n",
    "print(tokenizer2.decode(input2[\"input_ids\"][0], skip_special_tokens=False))"
   ],
   "id": "684c80f3d20be422",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'input_ids': tensor([[128000, 128006,   9125, 128007,    271,  38766,   1303,  33025,   2696,\n",
      "             25,   6790,    220,   2366,     18,    198,  15724,   2696,     25,\n",
      "            220,   1627,  10263,    220,   2366,     19,    271,   2675,    527,\n",
      "            264,  11190,  18328,     13, 128009, 128006,    882, 128007,    271,\n",
      "          36827,  35894, 113221,  21043, 115427, 118458,  11571, 128009, 128006,\n",
      "          78191, 128007,    271, 114880, 116382, 101426,   9554, 107471, 105644,\n",
      "          73686,  72718,   9554,   1811, 128009]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])}\n",
      "<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n",
      "\n",
      "Cutting Knowledge Date: December 2023\n",
      "Today Date: 26 Jul 2024\n",
      "\n",
      "You are a helpful assistant.<|eot_id|><|start_header_id|>user<|end_header_id|>\n",
      "\n",
      "天空为什么是蓝色的？<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n",
      "\n",
      "这是由于光的散射引起的。<|eot_id|>\n"
     ]
    }
   ],
   "execution_count": 133
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-09T05:59:21.497917Z",
     "start_time": "2025-05-09T05:59:21.492205Z"
    }
   },
   "cell_type": "code",
   "source": [
    "print(tokenizer2.bos_token)\n",
    "print(tokenizer2.eos_token)\n",
    "print(tokenizer2.special_tokens_map)\n",
    "print(tokenizer2.SPECIAL_TOKENS_ATTRIBUTES)\n",
    "print(tokenizer.all_special_tokens)"
   ],
   "id": "617dd5da308cf1dd",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<|begin_of_text|>\n",
      "<|eot_id|>\n",
      "{'bos_token': '<|begin_of_text|>', 'eos_token': '<|eot_id|>'}\n",
      "['bos_token', 'eos_token', 'unk_token', 'sep_token', 'pad_token', 'cls_token', 'mask_token', 'additional_special_tokens']\n",
      "['<|im_end|>', '<|endoftext|>', '<|im_start|>', '<|object_ref_start|>', '<|object_ref_end|>', '<|box_start|>', '<|box_end|>', '<|quad_start|>', '<|quad_end|>', '<|vision_start|>', '<|vision_end|>', '<|vision_pad|>', '<|image_pad|>', '<|video_pad|>']\n"
     ]
    }
   ],
   "execution_count": 134
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-09T06:49:57.881091Z",
     "start_time": "2025-05-09T06:49:57.876090Z"
    }
   },
   "cell_type": "code",
   "source": [
    "aa = [{'content': 'You are a helpful assistant.', 'role': 'system'}, {'content': '如何减少空气污染？ ', 'role': 'user'}, {'content': '''有很多方法来减少空气污染''', 'role': 'assistant'}]\n",
    "ab = tokenizer2.apply_chat_template(aa, tokenize=False)\n",
    "# print(ab)\n",
    "ac = tokenizer2(ab, return_tensors=\"pt\")\n",
    "print(ac)\n",
    "print(ac[\"input_ids\"])\n",
    "# print(len(ac['input_ids']))\n",
    "# print(tokenizer2.batch_decode(ac[\"input_ids\"]))"
   ],
   "id": "152359fe22abe9a5",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'input_ids': tensor([[128000, 128000, 128006,   9125, 128007,    271,  38766,   1303,  33025,\n",
      "           2696,     25,   6790,    220,   2366,     18,    198,  15724,   2696,\n",
      "             25,    220,   1627,  10263,    220,   2366,     19,    271,   2675,\n",
      "            527,    264,  11190,  18328,     13, 128009, 128006,    882, 128007,\n",
      "            271, 109425, 111689,  83747,  35894, 102146, 116028, 108208,  11571,\n",
      "         128009, 128006,  78191, 128007,    271,  19361, 112991,  41007,  37507,\n",
      "         111689,  83747,  35894, 102146, 116028, 108208, 128009]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])}\n",
      "tensor([[128000, 128000, 128006,   9125, 128007,    271,  38766,   1303,  33025,\n",
      "           2696,     25,   6790,    220,   2366,     18,    198,  15724,   2696,\n",
      "             25,    220,   1627,  10263,    220,   2366,     19,    271,   2675,\n",
      "            527,    264,  11190,  18328,     13, 128009, 128006,    882, 128007,\n",
      "            271, 109425, 111689,  83747,  35894, 102146, 116028, 108208,  11571,\n",
      "         128009, 128006,  78191, 128007,    271,  19361, 112991,  41007,  37507,\n",
      "         111689,  83747,  35894, 102146, 116028, 108208, 128009]])\n"
     ]
    }
   ],
   "execution_count": 145
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-09T06:50:48.841097Z",
     "start_time": "2025-05-09T06:50:48.836774Z"
    }
   },
   "cell_type": "code",
   "source": [
    "ab = tokenizer2.apply_chat_template(aa, tokenize=True, skip_special_tokens=True, return_tensors=\"pt\")\n",
    "print(ab)\n",
    "ab[0]\n"
   ],
   "id": "5e3b978f9d403e43",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[128000, 128006,   9125, 128007,    271,  38766,   1303,  33025,   2696,\n",
      "             25,   6790,    220,   2366,     18,    198,  15724,   2696,     25,\n",
      "            220,   1627,  10263,    220,   2366,     19,    271,   2675,    527,\n",
      "            264,  11190,  18328,     13, 128009, 128006,    882, 128007,    271,\n",
      "         109425, 111689,  83747,  35894, 102146, 116028, 108208,  11571, 128009,\n",
      "         128006,  78191, 128007,    271,  19361, 112991,  41007,  37507, 111689,\n",
      "          83747,  35894, 102146, 116028, 108208, 128009]])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([128000, 128006,   9125, 128007,    271,  38766,   1303,  33025,   2696,\n",
       "            25,   6790,    220,   2366,     18,    198,  15724,   2696,     25,\n",
       "           220,   1627,  10263,    220,   2366,     19,    271,   2675,    527,\n",
       "           264,  11190,  18328,     13, 128009, 128006,    882, 128007,    271,\n",
       "        109425, 111689,  83747,  35894, 102146, 116028, 108208,  11571, 128009,\n",
       "        128006,  78191, 128007,    271,  19361, 112991,  41007,  37507, 111689,\n",
       "         83747,  35894, 102146, 116028, 108208, 128009])"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 149
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-09T06:50:27.907633Z",
     "start_time": "2025-05-09T06:50:27.904175Z"
    }
   },
   "cell_type": "code",
   "source": "",
   "id": "bb291c645dfe11b0",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'b'"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 147
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
